Open
Conversation
Contributor
There was a problem hiding this comment.
Pull request overview
Adds a WebSocket heartbeat around Kubernetes exec sessions to reduce premature termination / stalled connections in “kubernetes mode” runners (Issue #228).
Changes:
- Introduces ping/pong heartbeat timers (configurable via env vars) for the exec WebSocket.
- Adds WebSocket lifecycle cleanup (close-with-timeout) on success/failure paths.
- Expands debug logging around
execPodStepand heartbeat behavior.
💡 Add Copilot custom instructions for smarter, more guided reviews. Learn how to get started.
Comment on lines
+261
to
+268
| const PING_PERIOD_MS = parseInt( | ||
| process.env.ACTIONS_RUNNER_HEARTBEAT_PERIOD_MS || '5000', | ||
| 10 | ||
| ) | ||
| const PING_READ_DEADLINE_MS = parseInt( | ||
| process.env.ACTIONS_RUNNER_HEARTBEAT_DEADLINE_MS || | ||
| String(PING_PERIOD_MS * 12 + 1000), | ||
| 10 |
Comment on lines
+310
to
+315
| pongTimeout = setTimeout(() => { | ||
| core.warning( | ||
| `[Heartbeat] No pong received in ${PING_READ_DEADLINE_MS}ms, connection may be stale` | ||
| ) | ||
| }, PING_READ_DEADLINE_MS) | ||
| } |
Comment on lines
+253
to
+258
| core.debug( | ||
| `[execPodStep] Starting execPodStep with command: ${JSON.stringify(command)}, podName: ${podName}, containerName: ${containerName}` | ||
| ) | ||
|
|
||
| command = fixArgs(command) | ||
| return await new Promise(function (resolve, reject) { | ||
| core.debug(`[execPodStep] Fixed command: ${JSON.stringify(command)}`) |
Comment on lines
+373
to
+376
| return new Promise<number>((resolve, reject) => { | ||
| core.debug('[execPodStep] About to call exec.exec') | ||
| let ws: any | null = null | ||
|
|
Comment on lines
+260
to
+371
| // Heartbeat constants matching kubectl's Go implementation | ||
| const PING_PERIOD_MS = parseInt( | ||
| process.env.ACTIONS_RUNNER_HEARTBEAT_PERIOD_MS || '5000', | ||
| 10 | ||
| ) | ||
| const PING_READ_DEADLINE_MS = parseInt( | ||
| process.env.ACTIONS_RUNNER_HEARTBEAT_DEADLINE_MS || | ||
| String(PING_PERIOD_MS * 12 + 1000), | ||
| 10 | ||
| ) | ||
| core.debug( | ||
| `[execPodStep] Heartbeat config: PING_PERIOD_MS=${PING_PERIOD_MS}, PING_READ_DEADLINE_MS=${PING_READ_DEADLINE_MS}` | ||
| ) | ||
|
|
||
| let pingInterval: ReturnType<typeof setTimeout> | null = null | ||
| let pongTimeout: ReturnType<typeof setTimeout> | null = null | ||
| let lastHeartbeatLog = 0 | ||
| const HEARTBEAT_LOG_INTERVAL_MS = 2 * 60 * 1000 // 2 minutes | ||
|
|
||
| const shouldLogHeartbeat = (): boolean => { | ||
| const now = Date.now() | ||
| if (now - lastHeartbeatLog >= HEARTBEAT_LOG_INTERVAL_MS) { | ||
| lastHeartbeatLog = now | ||
| return true | ||
| } | ||
| return false | ||
| } | ||
|
|
||
| const stopHeartbeat = (): void => { | ||
| if (shouldLogHeartbeat()) { | ||
| core.debug('[Heartbeat] stopHeartbeat called') | ||
| } | ||
| if (pingInterval) { | ||
| clearInterval(pingInterval) | ||
| pingInterval = null | ||
| } | ||
| if (pongTimeout) { | ||
| clearTimeout(pongTimeout) | ||
| pongTimeout = null | ||
| } | ||
| } | ||
|
|
||
| const resetPongTimeout = (): void => { | ||
| if (shouldLogHeartbeat()) { | ||
| core.debug('[Heartbeat] resetPongTimeout called') | ||
| } | ||
| if (pongTimeout) { | ||
| clearTimeout(pongTimeout) | ||
| pongTimeout = null | ||
| } | ||
| pongTimeout = setTimeout(() => { | ||
| core.warning( | ||
| `[Heartbeat] No pong received in ${PING_READ_DEADLINE_MS}ms, connection may be stale` | ||
| ) | ||
| }, PING_READ_DEADLINE_MS) | ||
| } | ||
|
|
||
| const startHeartbeat = (ws: any): void => { | ||
| core.debug( | ||
| `[Heartbeat] Starting with period=${PING_PERIOD_MS}ms, deadline=${PING_READ_DEADLINE_MS}ms` | ||
| ) | ||
| lastHeartbeatLog = Date.now() // Initialize timer | ||
|
|
||
| // Handle pong responses | ||
| ws.on('pong', () => { | ||
| if (shouldLogHeartbeat()) { | ||
| core.debug('[Heartbeat] Pong received') | ||
| } | ||
| resetPongTimeout() | ||
| }) | ||
|
|
||
| // Handle errors | ||
| ws.on('error', (err: Error) => { | ||
| core.error(`[Heartbeat] WebSocket error: ${err.message}`) | ||
| stopHeartbeat() | ||
| }) | ||
|
|
||
| // Cleanup on close | ||
| ws.on('close', () => { | ||
| core.debug('[Heartbeat] WebSocket closed, stopping heartbeat') | ||
| stopHeartbeat() | ||
| }) | ||
|
|
||
| // Set initial pong timeout | ||
| resetPongTimeout() | ||
|
|
||
| // Start ping loop | ||
| pingInterval = setInterval(() => { | ||
| // WebSocket readyState: 0 = CONNECTING, 1 = OPEN, 2 = CLOSING, 3 = CLOSED | ||
| if (shouldLogHeartbeat()) { | ||
| core.debug(`[Heartbeat] Ping loop, ws.readyState=${ws.readyState}`) | ||
| } | ||
| if (ws.readyState === 1) { | ||
| try { | ||
| ws.ping() | ||
| if (shouldLogHeartbeat()) { | ||
| core.debug('[Heartbeat] Ping sent') | ||
| } | ||
| } catch (err) { | ||
| core.error(`[Heartbeat] Ping failed: ${err}`) | ||
| stopHeartbeat() | ||
| } | ||
| } else { | ||
| if (shouldLogHeartbeat()) { | ||
| core.debug( | ||
| `[Heartbeat] WebSocket not open (readyState=${ws.readyState}), stopping` | ||
| ) | ||
| } | ||
| stopHeartbeat() | ||
| } | ||
| }, PING_PERIOD_MS) | ||
| } |
Comment on lines
+347
to
+369
| pingInterval = setInterval(() => { | ||
| // WebSocket readyState: 0 = CONNECTING, 1 = OPEN, 2 = CLOSING, 3 = CLOSED | ||
| if (shouldLogHeartbeat()) { | ||
| core.debug(`[Heartbeat] Ping loop, ws.readyState=${ws.readyState}`) | ||
| } | ||
| if (ws.readyState === 1) { | ||
| try { | ||
| ws.ping() | ||
| if (shouldLogHeartbeat()) { | ||
| core.debug('[Heartbeat] Ping sent') | ||
| } | ||
| } catch (err) { | ||
| core.error(`[Heartbeat] Ping failed: ${err}`) | ||
| stopHeartbeat() | ||
| } | ||
| } else { | ||
| if (shouldLogHeartbeat()) { | ||
| core.debug( | ||
| `[Heartbeat] WebSocket not open (readyState=${ws.readyState}), stopping` | ||
| ) | ||
| } | ||
| stopHeartbeat() | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.